#ifndef __ISER_H__
#define __ISER_H__

#include <infiniband/verbs.h>
#include <rdma/rdma_cma.h>

#include "iscsi.h"
#include "iser_hdr.h"
#include "rdma_event.h"

#define ISER_MAX_QUEUE_CMD      128     /* iSCSI cmd window size */
#define DEFAULT_RDMA_TRANSFER_SIZE  (512 * 1024)
#define DEFAULT_POOL_SIZE_MB    1024
#define MAX_CQ_ENTRIES          (MAX_QUEUE_CMD_MAX * DEFAULT_POOL_SIZE_MB)
#define USE_HUGE_PAGE 1

typedef void (*wc_handle_t)(struct ibv_wc *wc);

struct iser_portal {
        struct list_head iser_portal_siblings;
        int port;
        int af;
        struct rdma_cm_id *cma_listen_id;
};

enum task_flags {
        TASK_pending,
        TASK_in_scsi,
};

#define set_task_pending(t)        ((t)->flags |= (1 << TASK_pending))
#define clear_task_pending(t)      ((t)->flags &= ~(1 << TASK_pending))
#define task_pending(t)            ((t)->flags & (1 << TASK_pending))

#define set_task_in_scsi(t)        ((t)->flags |= (1 << TASK_in_scsi))
#define clear_task_in_scsi(t)      ((t)->flags &= ~(1 << TASK_in_scsi))
#define task_in_scsi(t)            ((t)->flags & (1 << TASK_in_scsi))

enum iser_ib_op_code {
        ISER_IB_RECV = 0,
        ISER_IB_SEND,
        ISER_IB_RDMA_WRITE,
        ISER_IB_RDMA_READ,
        ISER_IB_OP_END,
};

enum mgmt_req_result {
        MGMT_REQ_FAILED = -1,
        MGMT_REQ_DONE,
        MGMT_REQ_QUEUED,
};

/*
 * Work requests are either posted Receives for control messages,
 * or Send and RDMA ops (also considered "send" by IB)
 * They have different work request descriptors.
 * During a flush, we need to know the type of op and the
 * task to which it is related.
 */
struct iser_work_req {
        struct list_head wr_list;
        struct iser_task *task;
        enum iser_ib_op_code iser_ib_op;
        struct ibv_sge sge[2];
        union {
                struct ibv_recv_wr recv_wr;
                struct ibv_send_wr send_wr;
        };
};

/*
 * Pre-registered memory.  Buffers are allocated by iscsi from us, handed
 * to device to fill, then iser can send them directly without registration.
 * Also for write path.
 */
struct iser_membuf {
        void *addr;
        unsigned size;
        unsigned offset; /* offset within task data */
        struct list_head task_list;
        int rdma;
        buffer_t buf;
        struct list_head pool_list;
};

struct iser_pdu {
        struct iser_hdr *iser_hdr;
        struct iscsi_hdr *bhs;
        unsigned int ahssize;
        void *ahs;
        /* pdu data only, original buffer is reflected in ibv_sge */
        struct iser_membuf membuf;
};

/*
 * Each SCSI command may have its own RDMA parameters.  These appear on
 * the connection then later are assigned to the particular task to be
 * used when the target responds.
 */
struct iser_task {
        struct iser_conn *conn;

        struct iser_pdu pdu;

        struct iser_work_req rxd;
        struct iser_work_req txd;
        struct iser_work_req rdmad;

        int opcode;
        int is_immediate;
        int is_read;
        int is_write;
        int unsolicited;

        uint64_t tag;
        uint32_t cmd_sn;

        unsigned long flags;

        int in_len;
        int out_len;

        int unsol_sz;
        int unsol_remains;
        int rdma_rd_sz;
        int rdma_rd_remains;
        /* int rdma_rd_offset; // ToDo: multiple RDMA-Write buffers */
        int rdma_wr_sz;
        int rdma_wr_remains;

        /* read and write from the initiator's point of view */
        uint32_t rem_read_stag, rem_write_stag;
        uint64_t rem_read_va, rem_write_va;

        struct list_head in_buf_list;
        int in_buf_num;

        struct list_head out_buf_list;
        int out_buf_num;

        struct list_head exec_list;
        struct list_head rdma_list;
        struct list_head tx_list;
        struct list_head recv_list;

        /* linked to session->cmd_list */
        struct list_head session_list;

        struct list_head dout_task_list;

        int result;
        struct iscsi_cmd scmd;

        unsigned char *extdata;
};

enum iser_login_phase {
        LOGIN_PHASE_INIT,
        LOGIN_PHASE_START,      /* keep 1 send spot and 1 recv posted */
        LOGIN_PHASE_LAST_SEND,  /* need 1 more send before ff */
        LOGIN_PHASE_FF,         /* full feature */

        NUM_LOGIN_PHASE_VALS
};

/*
 * Parallels iscsi_connection.  Adds more fields for iser.
 */
struct iser_conn {
        struct iscsi_conn h;

        struct ibv_qp *qp_hndl;
        struct rdma_cm_id *cm_id;
        struct iser_device *dev;

        struct event_data sched_buf_alloc;
        struct list_head buf_alloc_list;

        struct event_data sched_rdma_rd;
        struct list_head rdma_rd_list;

        struct event_data sched_iosubmit;
        struct list_head iosubmit_list;

        struct event_data sched_tx;
        struct list_head resp_tx_list;

        struct list_head sent_list;

        struct event_data sched_post_recv;
        struct list_head post_recv_list;

        struct event_data sched_conn_free;

        struct sockaddr_storage peer_addr;  /* initiator address */
        char *peer_name;
        struct sockaddr_storage self_addr;  /* target address */
        char *self_name;

        unsigned int ssize, rsize, max_outst_pdu;

        /* FF resources */
        int ff_res_alloc;
        void *task_pool; /* iser_task structures */
        void *pdu_data_pool; /* memory for pdu, non-rdma send/recv */
        struct ibv_mr *pdu_data_mr;   /* memory registration for pdu data */
        struct ibv_mr *mr;   /* memory registration for pdu data */
        struct iser_task *nop_in_task;
        struct iser_task *text_tx_task;

        enum iser_login_phase login_phase;

        /* login phase resources, freed at full-feature */
        int login_res_alloc;
        void *login_task_pool;
        void *login_data_pool;
        struct ibv_mr *login_data_mr;
        struct iser_task *login_rx_task;
        struct iser_task *login_tx_task;
    
        void *private_mem;
        /* list of all iser conns */
        struct list_head conn_list;
	int  yield;
        int  core_reg;
	task_t  sched_task;
};

static inline struct iser_conn *ISER_CONN(struct iscsi_conn *iscsi_conn)
{
       return container_of(iscsi_conn, struct iser_conn, h);
}

struct iser_device {
        struct list_head list;
        struct ibv_context *ibv_ctxt;
        struct ibv_pd *pd;
        struct ibv_cq *cq;
        struct ibv_comp_channel *cq_channel;
        struct ibv_device_attr device_attr;
        
        /* membuf registered buffer, list area, handle */
        void *membuf_regbuf;
        void *membuf_listbuf;
        struct ibv_mr *membuf_mr;
        int waiting_for_mem;
        mem_handler_t *mem_handler;

        /* shared memory identifier */
        int rdma_hugetbl_shmid;

        struct event_data poll_sched;

        /* free and allocated membuf entries */
        struct list_head membuf_read_free, membuf_read_alloc;
        struct list_head membuf_write_free, membuf_write_alloc;

        struct ibv_mr *ff_resources_mr;
};

/* from iser.c */
int iser_init(int port, int *fd, void *private_mem);
void iser_polling(int hash, void *dev);

int iser_poll_cq(struct iser_device *dev, int max_polling);
void handle_wc(struct ibv_wc *wc);
void handle_wc_error(struct ibv_wc *wc);

/* from iser_conn.c */
int iser_conn_create(struct iser_conn **_conn);
void iser_conn_close(struct iser_conn *conn);
void iser_conn_force_close(struct iscsi_conn *iscsi_conn);
void iser_conn_free(void *conn);

void iser_conn_login_phase_set(struct iser_conn *conn,
                                      enum iser_login_phase phase);
int iser_conn_get(struct iser_conn *conn);
int iser_conn_getn(struct iser_conn *conn, int n);
void iser_conn_put(struct iser_conn *conn);
int iser_conn_show(struct iscsi_conn *iscsi_conn, char *buf,
                     int rest);

int iser_conn_getsockname(struct iscsi_conn *iscsi_conn,
                            struct sockaddr *sa, socklen_t *len);
int iser_conn_getpeername(struct iscsi_conn *iscsi_conn,
                            struct sockaddr *sa, socklen_t *len);

int iser_conn_alloc_login_resources(struct iser_conn *conn);
void iser_conn_free_login_resources(struct iser_conn *conn);

int iser_conn_alloc_ff_resources(struct iser_conn *conn);

int iser_ib_clear_tx_list(struct iser_conn *conn);

int iser_dev_post_recv(struct iser_conn *conn,
                       struct iser_task *task,
                       int num_recv_bufs);
int iser_dev_post_send(struct iser_conn *conn,
                       struct iser_work_req *iser_send,
                       int num_send_reqs);

/* from iser_device.c */
void iser_dev_init();
int iser_dev_create(struct iser_device **_dev, struct rdma_cm_id *cm_id);
void iser_dev_release(struct iser_device *dev);
int iser_dev_find(struct iser_device **dev, struct rdma_cm_id *cm_id);

int iser_dev_regmr(struct ibv_mr **_mr, struct ibv_pd *pd,
                        void *data_pool, unsigned long pool_size);
int iser_dev_deregmr(struct ibv_mr *mr);

void iser_dev_free_read_rdma_buf(struct iser_device *dev, struct iser_membuf *rdma_buf);
void iser_dev_free_write_rdma_buf(struct iser_device *dev, struct iser_membuf *rdma_buf);

struct iser_membuf *iser_dev_alloc_read_rdma_buf(struct iser_device *dev);
struct iser_membuf *iser_dev_alloc_write_rdma_buf(struct iser_device *dev);

/* from iser_task.c */
void iser_task_init(struct iser_task *task,
                           struct iser_conn *conn,
                           void *pdu_buf,
                           unsigned long buf_size,
                           struct ibv_mr *srmr);
void iser_task_unsolicited_init(struct iser_task *task,
                                       struct iser_conn *conn,
                                       void *pdu_data_buf,
                                       unsigned long buf_size,
                                       struct ibv_mr *srmr);

void iser_task_complete(struct iser_task *task);
int iser_task_queue(struct iscsi_session *session,
                           struct iser_task *task);
int iser_task_delivery(struct iser_task *task);

void iser_task_add_out_pdu_buf(struct iser_task *task,
                                      struct iser_membuf *data_buf,
                                      unsigned int offset);

int iser_task_alloc_rdma_bufs(struct iser_task *task);
void iser_task_free_dout_tasks(struct iser_task *task);

void iser_sched_buf_alloc(struct event_data *evt);
void iser_sched_iosubmit(struct event_data *evt);
void iser_sched_rdma_rd(struct event_data *evt);
void iser_sched_tx(struct event_data *evt);
void iser_sched_post_recv(struct event_data *evt);
void iser_sched_conn_free(struct event_data *evt);

/* from iser_sched.c */
void iser_sched_init();
int iser_exec_scheduled(void);

void iser_sched_init_event(struct event_data *evt,
                          sched_event_handler_t sched_handler, void *data);
void iser_sched_add_event(struct event_data *evt);
void iser_sched_remove_event(struct event_data *evt);

void schedule_task_iosubmit(struct iser_task *task, struct iser_conn *conn);
void schedule_rdma_read(struct iser_task *task, struct iser_conn *conn);
void schedule_resp_tx(struct iser_task *task, struct iser_conn *conn);
void schedule_post_recv(struct iser_task *task, struct iser_conn *conn);

/* from iser_text.c */
void iser_login_exec(struct iscsi_conn *iscsi_conn,
                     struct iser_pdu *rx_pdu,
                     struct iser_pdu *tx_pdu);
int iser_text_exec(struct iscsi_conn *iscsi_conn,
                   struct iser_pdu *rx_pdu,
                   struct iser_pdu *tx_pdu);

/* from iser_cmds.c */

void iser_cm_connect_request(struct rdma_cm_event *ev, void *core);
void iser_cm_conn_established(struct rdma_cm_event *ev, void *core);
void iser_cm_disconnected(struct rdma_cm_event *ev, void *core);
void iser_cm_timewait_exit(struct rdma_cm_event *ev, void *core);

void iser_login_rx(void *arg);
void iser_login_tx_complete(struct iser_conn *conn);
int iser_logout_exec(struct iser_task *task);

int iser_nop_out_rx(struct iser_task *task);
int iser_nop_out_exec(struct iser_task *task);
int iser_scsi_cmd_rx(struct iser_task *task);
int iser_data_out_rx(struct iser_task *dout_task);
int iser_tm_exec(struct iser_task *task);

void iser_scsi_cmd_iosubmit(struct iser_task *task, int not_last);
int iser_scsi_cmd_done(struct iscsi_cmd *scmd);

// void iser_acceptor(void);

/* from event.c */
int event_init();
void event_polling(int hash, void *dev);
void *event_loop(void *);
int event_add(int fd, int events, event_handle_t handler, void *data, void *private_mem);
void event_del(int fd);

void set_epoll_wait(int hash);
void clear_epoll_wait(int hash);

#endif  /* __ISER_H__ */
